{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "uuid": "9b6358d5-247c-46c6-8287-efbb3b848b31"
   },
   "outputs": [],
   "source": [
    "# !pip install pandas==0.24.2 --user\n",
    "# !pip install lightgbm==2.3.1 --user\n",
    "# !pip install xgboost==1.1.1 --user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "uuid": "6cb8b3d7-a557-41a5-809a-373bc3d4877c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tue Sep 22 01:45:16 2020       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 410.48                 Driver Version: 410.48                    |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla P100-PCIE...  Off  | 00000000:00:06.0 Off |                    0 |\n",
      "| N/A   29C    P0    32W / 250W |   1715MiB / 16280MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                       GPU Memory |\n",
      "|  GPU       PID   Type   Process name                             Usage      |\n",
      "|=============================================================================|\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'1.16.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "!nvidia-smi\n",
    "import numpy as np\n",
    "np.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "uuid": "2bbeec2d-6b98-4f33-97b6-8a78f575dd9e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              total        used        free      shared  buff/cache   available\n",
      "Mem:            30G        199M         30G         36M        374M         29G\n",
      "Swap:            0B          0B          0B\n"
     ]
    }
   ],
   "source": [
    "!free -h"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "uuid": "fe8b943e-ac18-4be1-b5e9-ad79d997ab22"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/bin/sh: line 0: cd: ./model: No such file or directory\n",
      "/data/nas/workspace/jupyter/load/model\n",
      "/data/nas/workspace/jupyter/load/model\n",
      "/data/nas/workspace/jupyter/load\n",
      "/data/nas/workspace/jupyter/load/model\n"
     ]
    }
   ],
   "source": [
    "# !cd ./model\n",
    "import  os\n",
    "\n",
    "print(os.getcwd())#获取当前工作目录路径\n",
    "print(os.path.abspath('.')) #获取当前工作目录路径\n",
    "# print os.path.abspath('test.txt') #获取当前目录文件下的工作目录路径\n",
    "print(os.path.abspath('..')) #获取当前工作的父目录 ！注意是父目录路径\n",
    "print(os.path.abspath(os.curdir))#获取当前工作目录路径"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    },
    "uuid": "70fa3a9f-826c-4af9-9c28-a320fd3261fa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11-16\n",
      "读取数据...\n",
      "train.shape (800000, 46)\n",
      "test.shape (200000, 45)\n",
      "初始拼接后： (1000000, 46)\n",
      "n特征处理后： (1000000, 52)\n",
      "count编码后： (1000000, 60)\n",
      "1data.shape (1000000, 52)\n",
      "2_data.shape (1000000, 56)\n",
      "预处理完毕 (1000000, 57)\n",
      "开始特征工程...\n",
      "data.shape (1000000, 73)\n",
      "开始模型训练...\n",
      "num0:mean_encode train.shape (800000, 83) (200000, 83)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num1:target_encode train.shape (800000, 88) (200000, 88)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [06:56<00:00, 83.34s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num2:target_encode train.shape (800000, 108) (200000, 108)\n",
      "输入数据维度： (800000, 103) (200000, 103)\n",
      "Current num of features: 101\n",
      "[0]\ttrain-auc:0.71042\teval-auc:0.70249\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7104181279375335,valid_auc0.7024893794968402\n",
      "[0.7024893794968402]\n",
      "[0]\ttrain-auc:0.70973\teval-auc:0.70376\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7097250296604438,valid_auc0.7037647897829287\n",
      "[0.7024893794968402, 0.7037647897829287]\n",
      "[0]\ttrain-auc:0.70996\teval-auc:0.70406\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7099643166750893,valid_auc0.7040573162032328\n",
      "[0.7024893794968402, 0.7037647897829287, 0.7040573162032328]\n",
      "[0]\ttrain-auc:0.71010\teval-auc:0.70203\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.71010263120682,valid_auc0.7020303844412975\n",
      "[0.7024893794968402, 0.7037647897829287, 0.7040573162032328, 0.7020303844412975]\n",
      "[0]\ttrain-auc:0.70986\teval-auc:0.70456\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7098637418542505,valid_auc0.7045624133077566\n",
      "[0.7024893794968402, 0.7037647897829287, 0.7040573162032328, 0.7020303844412975, 0.7045624133077566]\n",
      "all_auc: 0.5\n",
      "OOF-MEAN-AUC:0.703381, OOF-STD-AUC:0.000961\n",
      "Feature\n",
      "dti                                   29.60\n",
      "employmentTitle_target_skew           23.60\n",
      "postCode_target_std                   19.00\n",
      "rest_money_rate                       16.60\n",
      "title_pred_0                          13.60\n",
      "rest_Revol                            12.40\n",
      "n1                                     9.00\n",
      "homeOwnership_isDefault_kfold_mean     8.60\n",
      "grade                                  8.60\n",
      "regionCode_target_std                  8.20\n",
      "employmentLength                       6.80\n",
      "postCode_count                         6.80\n",
      "revolBal                               6.80\n",
      "homeOwnership_pred_0                   6.60\n",
      "term                                   6.40\n",
      "regionCode_pred_0                      6.20\n",
      "homeOwnership_target_skew              5.20\n",
      "n_feat_std                             5.00\n",
      "n7                                     4.80\n",
      "all_installment                        4.60\n",
      "dis_time                               4.40\n",
      "closeAcc                               4.40\n",
      "earliesCreditLine_Allmonth             3.80\n",
      "earliesCreditLine_year                 3.00\n",
      "n11                                    2.80\n",
      "employmentTitle_target_min             2.50\n",
      "openAcc                                2.20\n",
      "issueDate_month_count                  2.00\n",
      "loanAmnt                               2.00\n",
      "verificationStatus                     1.50\n",
      "n5                                     1.25\n",
      "applicationType                        1.20\n",
      "homeOwnership_pred_1                   1.00\n",
      "n12                                    1.00\n",
      "n13                                    1.00\n",
      "Name: importance, dtype: float64\n",
      "[0]\ttrain-auc:0.71197\teval-auc:0.70445\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7119712972918687,valid_auc0.7044496623396839\n",
      "[0.7044496623396839]\n",
      "[0]\ttrain-auc:0.71151\teval-auc:0.70696\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7115144040432397,valid_auc0.7069611121726944\n",
      "[0.7044496623396839, 0.7069611121726944]\n",
      "[0]\ttrain-auc:0.71238\teval-auc:0.70380\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7123763911856474,valid_auc0.7037968315611947\n",
      "[0.7044496623396839, 0.7069611121726944, 0.7037968315611947]\n",
      "[0]\ttrain-auc:0.71174\teval-auc:0.70590\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7117389557850893,valid_auc0.705903920477536\n",
      "[0.7044496623396839, 0.7069611121726944, 0.7037968315611947, 0.705903920477536]\n",
      "[0]\ttrain-auc:0.71207\teval-auc:0.70434\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7120713046911854,valid_auc0.7043367166794628\n",
      "[0.7044496623396839, 0.7069611121726944, 0.7037968315611947, 0.705903920477536, 0.7043367166794628]\n",
      "all_auc: 0.5\n",
      "OOF-MEAN-AUC:0.705090, OOF-STD-AUC:0.001167\n",
      "Feature\n",
      "issueDate_day                    23.800000\n",
      "installment                      20.600000\n",
      "homeOwnership_pred_0             17.800000\n",
      "n2                               17.200000\n",
      "total_income                     15.000000\n",
      "postCode_isDefault_kfold_mean    14.800000\n",
      "subGrade                         13.400000\n",
      "employmentTitle_pred_0           13.200000\n",
      "postCode_pred_0                  11.800000\n",
      "employmentTitle_pred_1            9.000000\n",
      "employmentTitle_target_std        7.600000\n",
      "regionCode_pred_0                 7.200000\n",
      "interestRate                      6.800000\n",
      "regionCode_target_skew            6.000000\n",
      "revolBal                          5.400000\n",
      "earliesCreditLine_Allmonth        4.800000\n",
      "n14                               4.800000\n",
      "n_feat_std                        4.600000\n",
      "title_pred_0                      4.200000\n",
      "n1                                4.000000\n",
      "n6                                3.400000\n",
      "employmentLength                  3.200000\n",
      "postCode_count                    3.000000\n",
      "totalAcc                          2.800000\n",
      "n5                                2.500000\n",
      "dis_time                          2.400000\n",
      "delinquency_2years                2.250000\n",
      "earliesCreditLine_year            1.800000\n",
      "purpose                           1.333333\n",
      "issueDate_week                    1.000000\n",
      "subGrade_count                    1.000000\n",
      "n8                                1.000000\n",
      "pubRecBankruptcies                1.000000\n",
      "applicationType                   1.000000\n",
      "Name: importance, dtype: float64\n",
      "[0]\ttrain-auc:0.71475\teval-auc:0.70749\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7147543404169187,valid_auc0.7074905821170885\n",
      "[0.7074905821170885]\n",
      "[0]\ttrain-auc:0.71447\teval-auc:0.70683\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7144718718730967,valid_auc0.706826077514229\n",
      "[0.7074905821170885, 0.706826077514229]\n",
      "[0]\ttrain-auc:0.71443\teval-auc:0.70671\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.714429855416528,valid_auc0.7067055148905892\n",
      "[0.7074905821170885, 0.706826077514229, 0.7067055148905892]\n",
      "[0]\ttrain-auc:0.71397\teval-auc:0.71075\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7139717572809304,valid_auc0.7107513725559202\n",
      "[0.7074905821170885, 0.706826077514229, 0.7067055148905892, 0.7107513725559202]\n",
      "[0]\ttrain-auc:0.71404\teval-auc:0.70847\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "train_auc:0.7140361337699601,valid_auc0.7084703882081897\n",
      "[0.7074905821170885, 0.706826077514229, 0.7067055148905892, 0.7107513725559202, 0.7084703882081897]\n",
      "all_auc: 0.5\n",
      "OOF-MEAN-AUC:0.708049, OOF-STD-AUC:0.001489\n",
      "Feature\n",
      "n2                               21.200000\n",
      "annualIncome                     20.400000\n",
      "homeOwnership_pred_0             19.600000\n",
      "employmentTitle_target_std       18.400000\n",
      "mean_interestRate                17.400000\n",
      "issueDate_year                   15.600000\n",
      "subGrade_count                   15.000000\n",
      "regionCode_pred_0                12.600000\n",
      "postCode_isDefault_kfold_mean     8.800000\n",
      "loanAmnt                          8.600000\n",
      "employmentLength                  7.600000\n",
      "employmentTitle_target_skew       7.400000\n",
      "postCode_target_std               7.000000\n",
      "postCode_pred_0                   6.800000\n",
      "postCode_target_skew              5.400000\n",
      "n_feat_sum                        5.400000\n",
      "revolBal                          4.200000\n",
      "dis_time                          4.200000\n",
      "grade                             4.000000\n",
      "avg_loanAmnt                      3.400000\n",
      "title_target_std                  3.400000\n",
      "title_pred_0                      3.200000\n",
      "closeAcc                          3.200000\n",
      "n8                                3.000000\n",
      "openAcc                           2.750000\n",
      "postCode_pred_1                   2.500000\n",
      "regionCode_count                  2.500000\n",
      "issueDate_month                   2.200000\n",
      "n_feat_max                        2.000000\n",
      "n4                                2.000000\n",
      "homeOwnership_pred_1              1.666667\n",
      "n6                                1.333333\n",
      "applicationType                   1.333333\n",
      "n0                                1.000000\n",
      "n5                                1.000000\n",
      "rest_pubRec                       1.000000\n",
      "Name: importance, dtype: float64\n",
      "整体指标得分： 0.7152001268524029\n"
     ]
    }
   ],
   "source": [
    "# coding: utf-8\n",
    "import multiprocessing\n",
    "from collections import Counter\n",
    "import xgboost as xgb\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import warnings\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "import lightgbm as lgb\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from tqdm import tqdm\n",
    "from sklearn.model_selection import KFold\n",
    "import gc\n",
    "from sklearn import preprocessing\n",
    "from scipy.stats import entropy\n",
    "# from imblearn.over_sampling import SMOTE\n",
    "# from imblearn.under_sampling import RandomUnderSampler\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn import metrics\n",
    "from sklearn.metrics import make_scorer, roc_auc_score\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "import datetime\n",
    "import time\n",
    "from itertools import product\n",
    "\n",
    "nowtime = datetime.date.today()\n",
    "nowtime = str(nowtime)[-5:]\n",
    "print(nowtime)\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "\n",
    "def load_dataset(DATA_PATH):\n",
    "    train_label = pd.read_csv(DATA_PATH + 'train.csv')['isDefault']\n",
    "    train = pd.read_csv(DATA_PATH + 'train.csv')\n",
    "    test = pd.read_csv(DATA_PATH + 'testA.csv')\n",
    "    del train['n2.1']\n",
    "    feats = [f for f in train.columns if f not in ['n_2.1', 'n2.2', 'n2.3', 'isDefault']]\n",
    "    # train = train[feats]\n",
    "    test = test[feats]\n",
    "    print('train.shape', train.shape)\n",
    "    print('test.shape', test.shape)\n",
    "\n",
    "    return train_label, train, test\n",
    "\n",
    "\n",
    "# 处理时间\n",
    "def transform_time(x):\n",
    "    day = int(x.split(' ')[0])\n",
    "    hour = int(x.split(' ')[2].split('.')[0].split(':')[0])\n",
    "    minute = int(x.split(' ')[2].split('.')[0].split(':')[1])\n",
    "    second = int(x.split(' ')[2].split('.')[0].split(':')[2])\n",
    "    return 86400 * day + 3600 * hour + 60 * minute + second\n",
    "\n",
    "\n",
    "def transform_day(date1):\n",
    "    date2 = \"2020-01-01\"\n",
    "    date1 = time.strptime(date1, \"%Y-%m-%d\")\n",
    "    date2 = time.strptime(date2, \"%Y-%m-%d\")\n",
    "\n",
    "    # 根据上面需要计算日期还是日期时间，来确定需要几个数组段。下标0表示年，小标1表示月，依次类推...\n",
    "    # date1=datetime.datetime(date1[0],date1[1],date1[2],date1[3],date1[4],date1[5])\n",
    "    # date2=datetime.datetime(date2[0],date2[1],date2[2],date2[3],date2[4],date2[5])\n",
    "    date1 = datetime.datetime(date1[0], date1[1], date1[2])\n",
    "    date2 = datetime.datetime(date2[0], date2[1], date2[2])\n",
    "    # 返回两个变量相差的值，就是相差天数\n",
    "    # print((date2 - date1).days)  # 将天数转成int型\n",
    "    return (date2 - date1).days\n",
    "\n",
    "\n",
    "# transform_day('2007-09-01')\n",
    "\n",
    "def labelEncoder_df(df, features):\n",
    "    for i in features:\n",
    "        encoder = preprocessing.LabelEncoder()\n",
    "        df[i] = encoder.fit_transform(df[i])\n",
    "\n",
    "\n",
    "\n",
    "class MeanEncoder:\n",
    "    def __init__(self, categorical_features, n_splits=5, target_type='classification', prior_weight_func=None):\n",
    "        \"\"\"\n",
    "        :param categorical_features: list of str, the name of the categorical columns to encode\n",
    "\n",
    "        :param n_splits: the number of splits used in mean encoding\n",
    "\n",
    "        :param target_type: str, 'regression' or 'classification'\n",
    "\n",
    "        :param prior_weight_func:\n",
    "        a function that takes in the number of observations, and outputs prior weight\n",
    "        when a dict is passed, the default exponential decay function will be used:\n",
    "        k: the number of observations needed for the posterior to be weighted equally as the prior\n",
    "        f: larger f --> smaller slope\n",
    "        \"\"\"\n",
    "\n",
    "        self.categorical_features = categorical_features\n",
    "        self.n_splits = n_splits\n",
    "        self.learned_stats = {}\n",
    "\n",
    "        if target_type == 'classification':\n",
    "            self.target_type = target_type\n",
    "            self.target_values = []\n",
    "        else:\n",
    "            self.target_type = 'regression'\n",
    "            self.target_values = None\n",
    "\n",
    "        if isinstance(prior_weight_func, dict):\n",
    "            self.prior_weight_func = eval('lambda x: 1 / (1 + np.exp((x - k) / f))', dict(prior_weight_func, np=np))\n",
    "        elif callable(prior_weight_func):\n",
    "            self.prior_weight_func = prior_weight_func\n",
    "        else:\n",
    "            self.prior_weight_func = lambda x: 1 / (1 + np.exp((x - 2) / 1))\n",
    "\n",
    "    @staticmethod\n",
    "    def mean_encode_subroutine(X_train, y_train, X_test, variable, target, prior_weight_func):\n",
    "        X_train = X_train[[variable]].copy()\n",
    "        X_test = X_test[[variable]].copy()\n",
    "\n",
    "        if target is not None:\n",
    "            nf_name = '{}_pred_{}'.format(variable, target)\n",
    "            X_train['pred_temp'] = (y_train == target).astype(int)  # classification\n",
    "        else:\n",
    "            nf_name = '{}_pred'.format(variable)\n",
    "            X_train['pred_temp'] = y_train  # regression\n",
    "        prior = X_train['pred_temp'].mean()\n",
    "\n",
    "        col_avg_y = X_train.groupby(by=variable, axis=0)['pred_temp'].agg({'mean': 'mean', 'beta': 'size'})\n",
    "        col_avg_y['beta'] = prior_weight_func(col_avg_y['beta'])\n",
    "        col_avg_y[nf_name] = col_avg_y['beta'] * prior + (1 - col_avg_y['beta']) * col_avg_y['mean']\n",
    "        col_avg_y.drop(['beta', 'mean'], axis=1, inplace=True)\n",
    "\n",
    "        nf_train = X_train.join(col_avg_y, on=variable)[nf_name].values\n",
    "        nf_test = X_test.join(col_avg_y, on=variable).fillna(prior, inplace=False)[nf_name].values\n",
    "\n",
    "        return nf_train, nf_test, prior, col_avg_y\n",
    "\n",
    "    def fit_transform(self, X, y):\n",
    "        \"\"\"\n",
    "        :param X: pandas DataFrame, n_samples * n_features\n",
    "        :param y: pandas Series or numpy array, n_samples\n",
    "        :return X_new: the transformed pandas DataFrame containing mean-encoded categorical features\n",
    "        \"\"\"\n",
    "        X_new = X.copy()\n",
    "        if self.target_type == 'classification':\n",
    "            skf = StratifiedKFold(self.n_splits)\n",
    "        else:\n",
    "            skf = KFold(self.n_splits)\n",
    "\n",
    "        if self.target_type == 'classification':\n",
    "            self.target_values = sorted(set(y))\n",
    "            self.learned_stats = {'{}_pred_{}'.format(variable, target): [] for variable, target in\n",
    "                                  product(self.categorical_features, self.target_values)}\n",
    "            for variable, target in product(self.categorical_features, self.target_values):\n",
    "                nf_name = '{}_pred_{}'.format(variable, target)\n",
    "                X_new.loc[:, nf_name] = np.nan\n",
    "                for large_ind, small_ind in skf.split(y, y):\n",
    "                    nf_large, nf_small, prior, col_avg_y = MeanEncoder.mean_encode_subroutine(\n",
    "                        X_new.iloc[large_ind], y.iloc[large_ind], X_new.iloc[small_ind], variable, target,\n",
    "                        self.prior_weight_func)\n",
    "                    X_new.iloc[small_ind, -1] = nf_small\n",
    "                    self.learned_stats[nf_name].append((prior, col_avg_y))\n",
    "        else:\n",
    "            self.learned_stats = {'{}_pred'.format(variable): [] for variable in self.categorical_features}\n",
    "            for variable in self.categorical_features:\n",
    "                nf_name = '{}_pred'.format(variable)\n",
    "                X_new.loc[:, nf_name] = np.nan\n",
    "                for large_ind, small_ind in skf.split(y, y):\n",
    "                    nf_large, nf_small, prior, col_avg_y = MeanEncoder.mean_encode_subroutine(\n",
    "                        X_new.iloc[large_ind], y.iloc[large_ind], X_new.iloc[small_ind], variable, None,\n",
    "                        self.prior_weight_func)\n",
    "                    X_new.iloc[small_ind, -1] = nf_small\n",
    "                    self.learned_stats[nf_name].append((prior, col_avg_y))\n",
    "        return X_new\n",
    "\n",
    "    def transform(self, X):\n",
    "        \"\"\"\n",
    "        :param X: pandas DataFrame, n_samples * n_features\n",
    "        :return X_new: the transformed pandas DataFrame containing mean-encoded categorical features\n",
    "        \"\"\"\n",
    "        X_new = X.copy()\n",
    "\n",
    "        if self.target_type == 'classification':\n",
    "            for variable, target in product(self.categorical_features, self.target_values):\n",
    "                nf_name = '{}_pred_{}'.format(variable, target)\n",
    "                X_new[nf_name] = 0\n",
    "                for prior, col_avg_y in self.learned_stats[nf_name]:\n",
    "                    X_new[nf_name] += X_new[[variable]].join(col_avg_y, on=variable).fillna(prior, inplace=False)[\n",
    "                        nf_name]\n",
    "                X_new[nf_name] /= self.n_splits\n",
    "        else:\n",
    "            for variable in self.categorical_features:\n",
    "                nf_name = '{}_pred'.format(variable)\n",
    "                X_new[nf_name] = 0\n",
    "                for prior, col_avg_y in self.learned_stats[nf_name]:\n",
    "                    X_new[nf_name] += X_new[[variable]].join(col_avg_y, on=variable).fillna(prior, inplace=False)[\n",
    "                        nf_name]\n",
    "                X_new[nf_name] /= self.n_splits\n",
    "\n",
    "        return X_new\n",
    "\n",
    "\n",
    "\n",
    "def employmentLength_deal(x):\n",
    "    if x == r'\\N':\n",
    "        result = -999\n",
    "    elif x == -999:\n",
    "        result = -999\n",
    "    elif x == '-999':\n",
    "        result = -999\n",
    "    elif x == '< 1 year':\n",
    "        result = 0.5\n",
    "    elif x == '10+ years':\n",
    "        result = 12\n",
    "    else:\n",
    "        result = int(x.split(' ')[0][0])\n",
    "    # print(result)\n",
    "    return result\n",
    "\n",
    "\n",
    "def earliesCreditLine_month_deal(x):\n",
    "    x = x.split('-')[0]\n",
    "    # print(x)\n",
    "    dict = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6, 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10,\n",
    "            'Nov': 11, 'Dec': 12}\n",
    "    result = dict[x]\n",
    "    return result\n",
    "\n",
    "\n",
    "def gradeTrans(x):\n",
    "    dict = {'A': 1, 'B': 2, 'C': 3, 'D': 4, 'E': 5, 'F': 6, 'G': 7}\n",
    "    result = dict[x]\n",
    "    return result\n",
    "\n",
    "\n",
    "def subGradeTrans(x):\n",
    "    dict = {'A': 1, 'B': 2, 'C': 3, 'D': 4, 'E': 5, 'F': 6, 'G': 7}\n",
    "    result = dict[x[0]]\n",
    "    result = result * 5 + int(x[1])\n",
    "    return result\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def myEntro(x):\n",
    "    \"\"\"\n",
    "        calculate shanno ent of x\n",
    "    \"\"\"\n",
    "    x = np.array(x)\n",
    "    x_value_list = set([x[i] for i in range(x.shape[0])])\n",
    "    ent = 0.0\n",
    "    for x_value in x_value_list:\n",
    "        p = float(x[x == x_value].shape[0]) / x.shape[0]\n",
    "        logp = np.log2(p)\n",
    "        ent -= p * logp\n",
    "    #     print(x_value,p,logp)\n",
    "    # print(ent)\n",
    "    return ent\n",
    "\n",
    "\n",
    "def myRms(records):\n",
    "    records = list(records)\n",
    "    \"\"\"\n",
    "    均方根值 反映的是有效值而不是平均值\n",
    "    \"\"\"\n",
    "    return np.math.sqrt(sum([x ** 2 for x in records]) / len(records))\n",
    "\n",
    "\n",
    "def myMode(x):\n",
    "    return np.mean(pd.Series.mode(x))\n",
    "\n",
    "\n",
    "def myQ25(x):\n",
    "    return x.quantile(0.25)\n",
    "\n",
    "\n",
    "def myQ75(x):\n",
    "    return x.quantile(0.75)\n",
    "\n",
    "\n",
    "def myRange(x):\n",
    "    return pd.Series.max(x) - pd.Series.min(x)\n",
    "\n",
    "\n",
    "# 预处理\n",
    "def data_preprocess(DATA_PATH):\n",
    "    train_label, train, test = load_dataset(DATA_PATH=DATA_PATH)\n",
    "    # 拼接数据\n",
    "\n",
    "    data = pd.concat([train, test], axis=0, ignore_index=True)\n",
    "    print('初始拼接后：', data.shape)\n",
    "    # n_feat = [f for f in data.columns if f[0] == 'n']\n",
    "\n",
    "    n_feat = ['n0', 'n1', 'n2', 'n4', 'n5', 'n6', 'n7', 'n8', 'n9', 'n10', 'n11', 'n12', 'n13', 'n14', ]\n",
    "    # nameList = ['min', 'max', 'sum', 'mean', 'median', 'skew', 'std', 'mode', 'range']\n",
    "    # statList = ['min', 'max', 'sum', 'mean', 'median', 'skew', 'std', myMode, myRange]\n",
    "    nameList = ['max', 'sum', 'mean', 'median', 'skew', 'std']\n",
    "    statList = ['max', 'sum', 'mean', 'median', 'skew', 'std']\n",
    "\n",
    "\n",
    "    for i in range(len(nameList)):\n",
    "        data['n_feat_{}'.format(nameList[i])] = data[n_feat].agg(statList[i], axis=1)\n",
    "    print('n特征处理后：', data.shape)\n",
    "\n",
    "\n",
    "    # count编码\n",
    "    count_list = ['subGrade', 'grade', 'postCode', 'regionCode','homeOwnership','title','employmentTitle','employmentLength']\n",
    "    data = count_coding(data, count_list)\n",
    "    print('count编码后：', data.shape)\n",
    "    ### 用数值特征对类别特征做统计刻画，随便挑了几个跟price相关性最高的匿名特征\n",
    "    cross_cat = ['subGrade', 'grade', 'employmentLength', 'term', 'homeOwnership', 'postCode', 'regionCode','employmentTitle','title']\n",
    "    cross_num = ['dti', 'revolBal','revolUtil', 'ficoRangeHigh', 'interestRate', 'loanAmnt', 'installment', 'annualIncome', 'n14',\n",
    "                 'n2', 'n6', 'n9', 'n5', 'n8']\n",
    "\n",
    "    data[['employmentLength']].fillna(-999, inplace=True)\n",
    "\n",
    "#     data = cross_cat_num(data, cross_num, cross_cat)  # 一阶交叉\n",
    "#     print('一阶特征处理后：', data.shape)\n",
    "#     data = cross_qua_cat_num(data)  # 二阶交叉\n",
    "#     print('二阶特征处理后：', data.shape)\n",
    "    # 缺失值处理\n",
    "    for temp in count_list:\n",
    "        del data[temp+'_count']\n",
    "    # num_fill_col = ['employmentLength', 'postCode', ]\n",
    "    cols = ['employmentTitle', 'employmentLength', 'postCode', 'dti', 'pubRecBankruptcies', 'revolUtil', 'title',\n",
    "            'n0', 'n1', 'n2', 'n4', 'n5', 'n6', 'n7', 'n8', 'n9', 'n10', 'n11', 'n12', 'n13', 'n14']\n",
    "    for col in cols:\n",
    "        data[col].fillna(r'\\N', inplace=True)\n",
    "    cols = [f for f in cols if f not in ['employmentLength']]\n",
    "    for col in cols:\n",
    "        data[col].replace({r'\\N': -999}, inplace=True)\n",
    "        data[col] = data[col]\n",
    "    # print('缺失值情况：', data.isnull().sum())\n",
    "\n",
    "    data['grade'] = data['grade'].apply(lambda x: gradeTrans(x))\n",
    "    data['subGrade'] = data['subGrade'].apply(lambda x: subGradeTrans(x))\n",
    "    print('1data.shape', data.shape)\n",
    "\n",
    "    data['employmentLength'] = data['employmentLength'].apply(lambda x: employmentLength_deal(x))\n",
    "    data['issueDate_year'] = data['issueDate'].apply(lambda x: int(x.split('-')[0]))\n",
    "    data['issueDate_month'] = data['issueDate'].apply(lambda x: int(x.split('-')[1]))\n",
    "    data['issueDate_day'] = data['issueDate'].apply(lambda x: transform_day(x))\n",
    "    data['issueDate_week'] = data['issueDate_day'].apply(lambda x: int(x % 7) + 1)\n",
    "\n",
    "    print('2_data.shape', data.shape)\n",
    "    data['earliesCreditLine_year'] = data['earliesCreditLine'].apply(lambda x: 2020 - (int(x.split('-')[-1])))\n",
    "    data['earliesCreditLine_month'] = data['earliesCreditLine'].apply(lambda x: earliesCreditLine_month_deal(x))\n",
    "    data['earliesCreditLine_Allmonth'] = data['earliesCreditLine_year'] * 12 - data['earliesCreditLine_month']\n",
    "    del data['issueDate'], data['earliesCreditLine']\n",
    "\n",
    "    print('预处理完毕', data.shape)\n",
    "\n",
    "    return data, train_label\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def kfold_stats_feature(train, test, feats, k):\n",
    "    folds = StratifiedKFold(n_splits=k, shuffle=True, random_state=6666)  # 这里最好和后面模型的K折交叉验证保持一致\n",
    "\n",
    "    train['fold'] = None\n",
    "    for fold_, (trn_idx, val_idx) in enumerate(folds.split(train, train['isDefault'])):\n",
    "        train.loc[val_idx, 'fold'] = fold_\n",
    "\n",
    "    kfold_features = []\n",
    "    for feat in feats:\n",
    "        nums_columns = ['isDefault']\n",
    "        for f in nums_columns:\n",
    "            colname = feat + '_' + f + '_kfold_mean'\n",
    "            kfold_features.append(colname)\n",
    "            train[colname] = None\n",
    "            for fold_, (trn_idx, val_idx) in enumerate(folds.split(train, train['isDefault'])):\n",
    "                tmp_trn = train.iloc[trn_idx]\n",
    "                order_label = tmp_trn.groupby([feat])[f].mean()\n",
    "                tmp = train.loc[train.fold == fold_, [feat]]\n",
    "                train.loc[train.fold == fold_, colname] = tmp[feat].map(order_label)\n",
    "                # fillna\n",
    "                global_mean = train[f].mean()\n",
    "                train.loc[train.fold == fold_, colname] = train.loc[train.fold == fold_, colname].fillna(global_mean)\n",
    "            train[colname] = train[colname].astype(float)\n",
    "\n",
    "        for f in nums_columns:\n",
    "            colname = feat + '_' + f + '_kfold_mean'\n",
    "            test[colname] = None\n",
    "            order_label = train.groupby([feat])[f].mean()\n",
    "            test[colname] = test[feat].map(order_label)\n",
    "            # fillna\n",
    "            global_mean = train[f].mean()\n",
    "            test[colname] = test[colname].fillna(global_mean)\n",
    "            test[colname] = test[colname].astype(float)\n",
    "    del train['fold']\n",
    "    return train, test\n",
    "\n",
    "def GridSearch(clf, params, X, y):\n",
    "    cscv = GridSearchCV(clf, params, scoring='roc_auc', n_jobs=4, cv=10)\n",
    "    cscv.fit(X, y)\n",
    "    print(cscv.cv_results_)\n",
    "    print(cscv.best_params_)\n",
    "    print(cscv.best_score_)\n",
    "\n",
    "### count编码\n",
    "def count_coding(df, fea_col):\n",
    "    for f in fea_col:\n",
    "        df[f + '_count'] = df[f].map(df[f].value_counts())\n",
    "    return (df)\n",
    "\n",
    "\n",
    "# 定义交叉特征统计\n",
    "def cross_cat_num(df, num_col, cat_col):\n",
    "    for f1 in tqdm(cat_col):\n",
    "        g = df.groupby(f1, as_index=False)\n",
    "        for f2 in tqdm(num_col):\n",
    "            feat = g[f2].agg({\n",
    "                '{}_{}_max'.format(f1, f2): 'max', '{}_{}_min'.format(f1, f2): 'min',\n",
    "                '{}_{}_median'.format(f1, f2): 'median',\n",
    "            })\n",
    "            df = df.merge(feat, on=f1, how='left')\n",
    "    return (df)\n",
    "\n",
    "\n",
    "def cross_qua_cat_num(df):\n",
    "    for f_pair in tqdm([\n",
    "        ['subGrade', 'regionCode'], ['grade', 'regionCode'], ['subGrade', 'postCode'], ['grade', 'postCode'], ['employmentTitle','title'],\n",
    "        ['regionCode','title'], ['postCode','title'], ['homeOwnership','title'], ['homeOwnership','employmentTitle'],['homeOwnership','employmentLength'],\n",
    "        ['regionCode', 'postCode']\n",
    "    ]):\n",
    "        ### 共现次数\n",
    "        df['_'.join(f_pair) + '_count'] = df.groupby(f_pair)['id'].transform('count')\n",
    "        ### n unique、熵\n",
    "        df = df.merge(df.groupby(f_pair[0], as_index=False)[f_pair[1]].agg({\n",
    "            '{}_{}_nunique'.format(f_pair[0], f_pair[1]): 'nunique',\n",
    "            '{}_{}_ent'.format(f_pair[0], f_pair[1]): lambda x: entropy(x.value_counts() / x.shape[0])\n",
    "        }), on=f_pair[0], how='left')\n",
    "        df = df.merge(df.groupby(f_pair[1], as_index=False)[f_pair[0]].agg({\n",
    "            '{}_{}_nunique'.format(f_pair[1], f_pair[0]): 'nunique',\n",
    "            '{}_{}_ent'.format(f_pair[1], f_pair[0]): lambda x: entropy(x.value_counts() / x.shape[0])\n",
    "        }), on=f_pair[1], how='left')\n",
    "        ### 比例偏好\n",
    "        df['{}_in_{}_prop'.format(f_pair[0], f_pair[1])] = df['_'.join(f_pair) + '_count'] / df[f_pair[1] + '_count']\n",
    "        df['{}_in_{}_prop'.format(f_pair[1], f_pair[0])] = df['_'.join(f_pair) + '_count'] / df[f_pair[0] + '_count']\n",
    "    return (df)\n",
    "\n",
    "\n",
    "### count编码\n",
    "def count_coding(df, fea_col):\n",
    "    for f in fea_col:\n",
    "        df[f + '_count'] = df[f].map(df[f].value_counts())\n",
    "    return (df)\n",
    "\n",
    "def gen_basicFea(data):\n",
    "    data['avg_income'] = data['annualIncome'] / data['employmentLength']\n",
    "    data['total_income'] = data['annualIncome'] * data['employmentLength']\n",
    "    data['avg_loanAmnt'] = data['loanAmnt'] / data['term']\n",
    "    data['mean_interestRate'] = data['interestRate'] / data['term']\n",
    "    data['all_installment'] = data['installment'] * data['term']\n",
    "\n",
    "    data['rest_money_rate'] = data['avg_loanAmnt'] / (data['annualIncome'] + 0.1)  # 287个收入为0\n",
    "    data['rest_money'] = data['annualIncome'] - data['avg_loanAmnt']\n",
    "\n",
    "    data['closeAcc'] = data['totalAcc'] - data['openAcc']\n",
    "    data['ficoRange_mean'] = (data['ficoRangeHigh'] + data['ficoRangeLow']) / 2\n",
    "    del data['ficoRangeHigh'], data['ficoRangeLow']\n",
    "\n",
    "    data['rest_pubRec'] = data['pubRec'] - data['pubRecBankruptcies']\n",
    "\n",
    "    data['rest_Revol'] = data['loanAmnt'] - data['revolBal']\n",
    "\n",
    "    data['dis_time'] = data['issueDate_year'] - (2020 - data['earliesCreditLine_year'])\n",
    "    for col in ['employmentTitle', 'grade', 'subGrade', 'regionCode', 'issueDate_month', 'postCode']:\n",
    "        data['{}_count'.format(col)] = data.groupby([col])['id'].transform('count')\n",
    "\n",
    "    return data\n",
    "\n",
    "\n",
    "def plotroc(train_y, train_pred, test_y, val_pred):\n",
    "    lw = 2\n",
    "    ##train\n",
    "    fpr, tpr, thresholds = roc_curve(train_y.values, train_pred, pos_label=1.0)\n",
    "    train_auc_value = roc_auc_score(train_y.values, train_pred)\n",
    "    ##valid\n",
    "    fpr, tpr, thresholds = roc_curve(test_y.values, val_pred, pos_label=1.0)\n",
    "    valid_auc_value = roc_auc_score(test_y.values, val_pred)\n",
    "\n",
    "    return train_auc_value, valid_auc_value\n",
    "\n",
    "\n",
    "def xgb_model(train, target, test, k):\n",
    "    # saveFeature_df = pd.read_csv('../feature/xgb_920_556_5_score.csv',header=None)\n",
    "#     saveFeature_df2 = pd.read_csv('../feature/xgb_09-20_74_0.8_0.6_5_score.csv',header=None)\n",
    "    \n",
    "    # saveFeature_df.columns=['feature','score']\n",
    "#     saveFeature_df2.columns=['feature','score']\n",
    "\n",
    "#     saveFeature_list = list(saveFeature_df['feature'].values)\n",
    "#     saveFeature_list2 = list(saveFeature_df2['feature'].values)\n",
    "    \n",
    "    # saveFeature_list = list(saveFeature_df[saveFeature_df['score']>10]['feature'])\n",
    "    saveFeature_list=list(train.columns)\n",
    "    feats = [f for f in saveFeature_list if f not in ['id', 'isDefault']]\n",
    "    feaNum = len(feats)\n",
    "    print('Current num of features:', len(feats))\n",
    "\n",
    "    seeds = [2020,666666,188888]\n",
    "    output_preds = 0\n",
    "    xgb_oof_probs = np.zeros(train.shape[0])\n",
    "\n",
    "    for seed in seeds:\n",
    "        folds = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)\n",
    "        oof_probs = np.zeros(train.shape[0])\n",
    "\n",
    "        offline_score = []\n",
    "        feature_importance_df = pd.DataFrame()\n",
    "        params = {'booster': 'gbtree',\n",
    "                  'objective': 'binary:logistic',\n",
    "                  'eval_metric': 'auc',\n",
    "                  'min_child_weight': 5,\n",
    "                  'max_depth': 8,\n",
    "                  'subsample': ss,\n",
    "                  'colsample_bytree': fs,\n",
    "                  'eta': 0.01,\n",
    "\n",
    "                  'seed': seed,\n",
    "                  'nthread': -1,\n",
    "\n",
    "                  'tree_method': 'gpu_hist'\n",
    "                  }\n",
    "        for i, (train_index, test_index) in enumerate(folds.split(train, target)):\n",
    "            \n",
    "            train_y, test_y = target[train_index], target[test_index]\n",
    "            train_X, test_X = train[feats].iloc[train_index, :], train[feats].iloc[test_index, :]\n",
    "            train_matrix = xgb.DMatrix(train_X, label=train_y, missing=np.nan)\n",
    "            valid_matrix = xgb.DMatrix(test_X, label=test_y, missing=np.nan)\n",
    "            test_matrix = xgb.DMatrix(test[feats], missing=np.nan)\n",
    "            watchlist = [(train_matrix, 'train'), (valid_matrix, 'eval')]\n",
    "            model = xgb.train(params, train_matrix, num_boost_round=100000, evals=watchlist, verbose_eval=100,\n",
    "                              early_stopping_rounds=600)\n",
    "            val_pred = model.predict(valid_matrix, ntree_limit=model.best_ntree_limit)\n",
    "            train_pred = model.predict(train_matrix, ntree_limit=model.best_ntree_limit)\n",
    "            xgb_oof_probs[test_index] += val_pred / len(seeds)\n",
    "            # oof_probs[test_index] += val_pred\n",
    "            test_pred = model.predict(test_matrix, ntree_limit=model.best_ntree_limit)\n",
    "\n",
    "            # 绘制roc曲线\n",
    "            train_auc_value, valid_auc_value = plotroc(train_y, train_pred, test_y, val_pred)\n",
    "            print('train_auc:{},valid_auc{}'.format(train_auc_value, valid_auc_value))\n",
    "            offline_score.append(valid_auc_value)\n",
    "            print(offline_score)\n",
    "            output_preds += test_pred / k / len(seeds)\n",
    "            \n",
    "#             sub_df = test[['id']].copy()\n",
    "#             sub_df['isDefault'] = output_preds\n",
    "#             off = test[['id']].copy()\n",
    "#             subVal_df = train[['id']].copy()\n",
    "#             subVal_df.loc[test_index,'isDefault'] = xgb_oof_probs[test_index]\n",
    "#             outpath = '../user_data/fold/'\n",
    "#             fold_score = round(valid_auc_value, 5)\n",
    "#             sub_df.to_csv( outpath + str(fold_score) + '_' + str(feaNum) + '_' + nowtime + '_{}_{}_xgb.csv'.format(i, kflod_num),\n",
    "#             index=False)\n",
    "#             subVal_df.to_csv(outpath + str(fold_score) + '_' + str(feaNum) + '_' + nowtime + '_{}_{}_xgbVal.csv'.format(i, kflod_num),\n",
    "#             index=False)\n",
    "\n",
    "            fold_importance_df = pd.DataFrame()\n",
    "            fold_importance_df[\"Feature\"] = model.get_fscore().keys()\n",
    "            fold_importance_df[\"importance\"] = model.get_fscore().values()\n",
    "            fold_importance_df[\"fold\"] = i + 1\n",
    "\n",
    "            feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n",
    "#             print(feature_importance_df.groupby(['Feature'])['importance'].mean().sort_values(ascending=False).head(50))\n",
    "#             feature_sorted = feature_importance_df.groupby(['Feature'])['importance'].mean().sort_values(ascending=False)\n",
    "#             feature_sorted.to_csv('../feature/xgb_{}_{}_{}_{}_score.csv'.format(i+1,nowtime, feaNum, kflod_num))\n",
    "\n",
    "#             if i==3:\n",
    "#                 break\n",
    "#             gc.collect()\n",
    "\n",
    "\n",
    "        print('all_auc:', roc_auc_score(target.values, oof_probs))\n",
    "        print('OOF-MEAN-AUC:%.6f, OOF-STD-AUC:%.6f' % (np.mean(offline_score), np.std(offline_score)))\n",
    "        feature_sorted = feature_importance_df.groupby(['Feature'])['importance'].mean().sort_values(ascending=False)\n",
    "        feature_sorted.to_csv('../feature/xgb_importance.csv')\n",
    "        top_features = feature_sorted.index\n",
    "        print(feature_importance_df.groupby(['Feature'])['importance'].mean().sort_values(ascending=False).head(50))\n",
    "    return output_preds, xgb_oof_probs, np.mean(offline_score), feaNum\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    DATA_PATH = '../data/'\n",
    "    print('读取数据...')\n",
    "    data, train_label = data_preprocess(DATA_PATH=DATA_PATH)\n",
    "\n",
    "    print('开始特征工程...')\n",
    "    data = gen_basicFea(data)\n",
    "\n",
    "\n",
    "    print('data.shape', data.shape)\n",
    "    print('开始模型训练...')\n",
    "    train = data[~data['isDefault'].isnull()].copy()\n",
    "    target = train_label\n",
    "    test = data[data['isDefault'].isnull()].copy()\n",
    "\n",
    "    target_encode_cols = ['postCode', 'regionCode', 'homeOwnership', 'employmentTitle','title']\n",
    "\n",
    "    kflod_num = 5\n",
    "    ss = 0.8\n",
    "    fs = 0.4\n",
    "\n",
    "    class_list = ['postCode', 'regionCode', 'homeOwnership', 'employmentTitle','title']\n",
    "    MeanEnocodeFeature = class_list  # 声明需要平均数编码的特征\n",
    "    ME = MeanEncoder(MeanEnocodeFeature, target_type='classification')  # 声明平均数编码的类\n",
    "    train = ME.fit_transform(train, target)  # 对训练数据集的X和y进行拟合\n",
    "    # x_train_fav = ME.fit_transform(x_train,y_train_fav)#对训练数据集的X和y进行拟合\n",
    "    test = ME.transform(test)  # 对测试集进行编码\n",
    "    print('num0:mean_encode train.shape', train.shape, test.shape)\n",
    "\n",
    "    train, test = kfold_stats_feature(train, test, target_encode_cols, kflod_num)\n",
    "    print('num1:target_encode train.shape', train.shape, test.shape)\n",
    "    ### target encoding目标编码，回归场景相对来说做目标编码的选择更多，不仅可以做均值编码，还可以做标准差编码、中位数编码等\n",
    "    enc_cols = []\n",
    "    stats_default_dict = {\n",
    "        'max': train['isDefault'].max(),\n",
    "        'min': train['isDefault'].min(),\n",
    "        'median': train['isDefault'].median(),\n",
    "        'mean': train['isDefault'].mean(),\n",
    "        'sum': train['isDefault'].sum(),\n",
    "        'std': train['isDefault'].std(),\n",
    "        'skew': train['isDefault'].skew(),\n",
    "        'kurt': train['isDefault'].kurt(),\n",
    "        'mad': train['isDefault'].mad()\n",
    "    }\n",
    "    ### 暂且选择这三种编码\n",
    "    enc_stats = ['max', 'min', 'skew', 'std']\n",
    "    skf = KFold(n_splits=kflod_num, shuffle=True, random_state=6666)\n",
    "    for f in tqdm(['postCode', 'regionCode', 'homeOwnership', 'employmentTitle','title']):\n",
    "        enc_dict = {}\n",
    "        for stat in enc_stats:\n",
    "            enc_dict['{}_target_{}'.format(f, stat)] = stat\n",
    "            train['{}_target_{}'.format(f, stat)] = 0\n",
    "            test['{}_target_{}'.format(f, stat)] = 0\n",
    "            enc_cols.append('{}_target_{}'.format(f, stat))\n",
    "        for i, (trn_idx, val_idx) in enumerate(skf.split(train, target)):\n",
    "            trn_x, val_x = train.iloc[trn_idx].reset_index(drop=True), train.iloc[val_idx].reset_index(drop=True)\n",
    "            enc_df = trn_x.groupby(f, as_index=False)['isDefault'].agg(enc_dict)\n",
    "            val_x = val_x[[f]].merge(enc_df, on=f, how='left')\n",
    "            test_x = test[[f]].merge(enc_df, on=f, how='left')\n",
    "            for stat in enc_stats:\n",
    "                val_x['{}_target_{}'.format(f, stat)] = val_x['{}_target_{}'.format(f, stat)].fillna(\n",
    "                    stats_default_dict[stat])\n",
    "                test_x['{}_target_{}'.format(f, stat)] = test_x['{}_target_{}'.format(f, stat)].fillna(\n",
    "                    stats_default_dict[stat])\n",
    "                train.loc[val_idx, '{}_target_{}'.format(f, stat)] = val_x['{}_target_{}'.format(f, stat)].values\n",
    "                test['{}_target_{}'.format(f, stat)] += test_x['{}_target_{}'.format(f, stat)].values / skf.n_splits\n",
    "\n",
    "    print('num2:target_encode train.shape', train.shape, test.shape)\n",
    "\n",
    "    train.drop(['postCode', 'regionCode', 'homeOwnership', 'employmentTitle','title'], axis=1, inplace=True)\n",
    "    test.drop(['postCode', 'regionCode', 'homeOwnership', 'employmentTitle','title'], axis=1, inplace=True)\n",
    "    print('输入数据维度：', train.shape, test.shape)\n",
    "    \n",
    "    xgb_preds, xgb_oof, xgb_score, feaNum = xgb_model(train=train, target=target, test=test, k=kflod_num)\n",
    "\n",
    "    lgb_score = round(xgb_score, 5)\n",
    "    sub_df = test[['id']].copy()\n",
    "    sub_df['isDefault'] = xgb_preds\n",
    "    off = test[['id']].copy()\n",
    "    subVal_df = train[['id']].copy()\n",
    "    subVal_df['isDefault'] = xgb_oof\n",
    "    outpath = '../user_data/'\n",
    "\n",
    "    all_auc_score = roc_auc_score(train_label, subVal_df['isDefault'])\n",
    "    print('整体指标得分：', all_auc_score)\n",
    "    all_auc_score = round(all_auc_score, 5)\n",
    "\n",
    "    sub_df.to_csv(outpath+'xgb1.csv',index=False)\n",
    "    subVal_df.to_csv(outpath+'xgb1Val.csv',index=False)\n",
    "    # sub_df.to_csv(\n",
    "    #     outpath + str(all_auc_score) + '_' + str(feaNum) + '_' + nowtime + '_{}_{}_{}_xgb.csv'.format(ss, fs,\n",
    "    #                                                                                                   kflod_num),\n",
    "    #     index=False)\n",
    "    # subVal_df.to_csv(\n",
    "    #     outpath + str(all_auc_score) + '_' + str(feaNum) + '_' + nowtime + '_{}_{}_{}_subVal.csv'.format(ss, fs,\n",
    "    #                                                                                                      kflod_num),\n",
    "    #     index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "uuid": "5c753d72-6075-4935-8646-c095c552e062"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data.shape (1000000, 73)\n",
      "开始模型训练...\n",
      "num0:mean_encode train.shape (800000, 83) (200000, 83)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/5 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num1:target_encode train.shape (800000, 88) (200000, 88)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [06:48<00:00, 81.79s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num2:target_encode train.shape (800000, 108) (200000, 108)\n",
      "输入数据维度： (800000, 103) (200000, 103)\n",
      "Current num of features: 101\n",
      "[0]\ttrain-auc:0.71042\teval-auc:0.70249\n",
      "Multiple eval metrics have been passed: 'eval-auc' will be used for early stopping.\n",
      "\n",
      "Will train until eval-auc hasn't improved in 600 rounds.\n",
      "[100]\ttrain-auc:0.73574\teval-auc:0.72394\n",
      "[200]\ttrain-auc:0.74159\teval-auc:0.72728\n",
      "[300]\ttrain-auc:0.74725\teval-auc:0.73018\n",
      "[400]\ttrain-auc:0.75279\teval-auc:0.73265\n",
      "[500]\ttrain-auc:0.75749\teval-auc:0.73440\n",
      "[600]\ttrain-auc:0.76170\teval-auc:0.73565\n",
      "[700]\ttrain-auc:0.76547\teval-auc:0.73649\n",
      "[800]\ttrain-auc:0.76903\teval-auc:0.73721\n",
      "[900]\ttrain-auc:0.77252\teval-auc:0.73775\n",
      "[1000]\ttrain-auc:0.77565\teval-auc:0.73820\n",
      "[1100]\ttrain-auc:0.77849\teval-auc:0.73857\n",
      "[1200]\ttrain-auc:0.78131\teval-auc:0.73884\n",
      "[1300]\ttrain-auc:0.78404\teval-auc:0.73909\n",
      "[1400]\ttrain-auc:0.78673\teval-auc:0.73930\n",
      "[1500]\ttrain-auc:0.78935\teval-auc:0.73949\n",
      "[1600]\ttrain-auc:0.79181\teval-auc:0.73965\n",
      "[1700]\ttrain-auc:0.79428\teval-auc:0.73982\n",
      "[1800]\ttrain-auc:0.79668\teval-auc:0.73996\n",
      "[1900]\ttrain-auc:0.79909\teval-auc:0.74010\n",
      "[2000]\ttrain-auc:0.80134\teval-auc:0.74021\n",
      "[2100]\ttrain-auc:0.80350\teval-auc:0.74033\n",
      "[2200]\ttrain-auc:0.80559\teval-auc:0.74042\n",
      "[2300]\ttrain-auc:0.80778\teval-auc:0.74049\n",
      "[2400]\ttrain-auc:0.80995\teval-auc:0.74055\n",
      "[2500]\ttrain-auc:0.81212\teval-auc:0.74060\n",
      "[2600]\ttrain-auc:0.81420\teval-auc:0.74065\n",
      "[2700]\ttrain-auc:0.81623\teval-auc:0.74072\n",
      "[2800]\ttrain-auc:0.81831\teval-auc:0.74078\n",
      "[2900]\ttrain-auc:0.82043\teval-auc:0.74083\n",
      "[3000]\ttrain-auc:0.82230\teval-auc:0.74087\n"
     ]
    }
   ],
   "source": [
    "def xgb_model(train, target, test, k):\n",
    "    saveFeature_df = pd.read_csv('../feature/xgb_importance.csv',header=None)\n",
    "#     saveFeature_df2 = pd.read_csv('../feature/xgb_09-20_74_0.8_0.6_5_score.csv',header=None)\n",
    "    \n",
    "    saveFeature_df.columns=['feature','score']\n",
    "#     saveFeature_df2.columns=['feature','score']\n",
    "\n",
    "#     saveFeature_list = list(saveFeature_df['feature'].values)\n",
    "#     saveFeature_list2 = list(saveFeature_df2['feature'].values)\n",
    "    \n",
    "    saveFeature_list = list(saveFeature_df[saveFeature_df['score']>0]['feature'])\n",
    "    saveFeature_list=list(train.columns)\n",
    "    feats = [f for f in saveFeature_list if f not in ['id', 'isDefault']]\n",
    "    feaNum = len(feats)\n",
    "    print('Current num of features:', len(feats))\n",
    "\n",
    "    seeds = [2020,666666,188888]\n",
    "    output_preds = 0\n",
    "    xgb_oof_probs = np.zeros(train.shape[0])\n",
    "\n",
    "    for seed in seeds:\n",
    "        folds = StratifiedKFold(n_splits=k, shuffle=True, random_state=seed)\n",
    "        oof_probs = np.zeros(train.shape[0])\n",
    "\n",
    "        offline_score = []\n",
    "        feature_importance_df = pd.DataFrame()\n",
    "        params = {'booster': 'gbtree',\n",
    "                  'objective': 'binary:logistic',\n",
    "                  'eval_metric': 'auc',\n",
    "                  'min_child_weight': 5,\n",
    "                  'max_depth': 8,\n",
    "                  'subsample': ss,\n",
    "                  'colsample_bytree': fs,\n",
    "                  'eta': 0.01,\n",
    "\n",
    "                  'seed': seed,\n",
    "                  'nthread': -1,\n",
    "\n",
    "                  'tree_method': 'gpu_hist'\n",
    "                  }\n",
    "        for i, (train_index, test_index) in enumerate(folds.split(train, target)):\n",
    "            \n",
    "            train_y, test_y = target[train_index], target[test_index]\n",
    "            train_X, test_X = train[feats].iloc[train_index, :], train[feats].iloc[test_index, :]\n",
    "            train_matrix = xgb.DMatrix(train_X, label=train_y, missing=np.nan)\n",
    "            valid_matrix = xgb.DMatrix(test_X, label=test_y, missing=np.nan)\n",
    "            test_matrix = xgb.DMatrix(test[feats], missing=np.nan)\n",
    "            watchlist = [(train_matrix, 'train'), (valid_matrix, 'eval')]\n",
    "            model = xgb.train(params, train_matrix, num_boost_round=100000, evals=watchlist, verbose_eval=100,\n",
    "                              early_stopping_rounds=600)\n",
    "            val_pred = model.predict(valid_matrix, ntree_limit=model.best_ntree_limit)\n",
    "            train_pred = model.predict(train_matrix, ntree_limit=model.best_ntree_limit)\n",
    "            xgb_oof_probs[test_index] += val_pred / len(seeds)\n",
    "            # oof_probs[test_index] += val_pred\n",
    "            test_pred = model.predict(test_matrix, ntree_limit=model.best_ntree_limit)\n",
    "\n",
    "            # 绘制roc曲线\n",
    "            train_auc_value, valid_auc_value = plotroc(train_y, train_pred, test_y, val_pred)\n",
    "            print('train_auc:{},valid_auc{}'.format(train_auc_value, valid_auc_value))\n",
    "            offline_score.append(valid_auc_value)\n",
    "            print(offline_score)\n",
    "            output_preds += test_pred / k / len(seeds)\n",
    "            \n",
    "#             sub_df = test[['id']].copy()\n",
    "#             sub_df['isDefault'] = output_preds\n",
    "#             off = test[['id']].copy()\n",
    "#             subVal_df = train[['id']].copy()\n",
    "#             subVal_df.loc[test_index,'isDefault'] = xgb_oof_probs[test_index]\n",
    "#             outpath = '../user_data/fold/'\n",
    "#             fold_score = round(valid_auc_value, 5)\n",
    "#             sub_df.to_csv( outpath + str(fold_score) + '_' + str(feaNum) + '_' + nowtime + '_{}_{}_xgb.csv'.format(i, kflod_num),\n",
    "#             index=False)\n",
    "#             subVal_df.to_csv(outpath + str(fold_score) + '_' + str(feaNum) + '_' + nowtime + '_{}_{}_xgbVal.csv'.format(i, kflod_num),\n",
    "#             index=False)\n",
    "\n",
    "            fold_importance_df = pd.DataFrame()\n",
    "            fold_importance_df[\"Feature\"] = model.get_fscore().keys()\n",
    "            fold_importance_df[\"importance\"] = model.get_fscore().values()\n",
    "            fold_importance_df[\"fold\"] = i + 1\n",
    "\n",
    "            feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n",
    "#             print(feature_importance_df.groupby(['Feature'])['importance'].mean().sort_values(ascending=False).head(50))\n",
    "#             feature_sorted = feature_importance_df.groupby(['Feature'])['importance'].mean().sort_values(ascending=False)\n",
    "#             feature_sorted.to_csv('../feature/xgb_{}_{}_{}_{}_score.csv'.format(i+1,nowtime, feaNum, kflod_num))\n",
    "\n",
    "#             if i==3:\n",
    "#                 break\n",
    "#             gc.collect()\n",
    "\n",
    "\n",
    "        print('all_auc:', roc_auc_score(target.values, oof_probs))\n",
    "        print('OOF-MEAN-AUC:%.6f, OOF-STD-AUC:%.6f' % (np.mean(offline_score), np.std(offline_score)))\n",
    "        feature_sorted = feature_importance_df.groupby(['Feature'])['importance'].mean().sort_values(ascending=False)\n",
    "#         feature_sorted.to_csv('../feature/xgb_importance.csv')\n",
    "        top_features = feature_sorted.index\n",
    "        print(feature_importance_df.groupby(['Feature'])['importance'].mean().sort_values(ascending=False).head(50))\n",
    "    return output_preds, xgb_oof_probs, np.mean(offline_score), feaNum\n",
    "\n",
    "\n",
    "\n",
    "del train,test\n",
    "gc.collect()\n",
    "print('data.shape', data.shape)\n",
    "print('开始模型训练...')\n",
    "train = data[~data['isDefault'].isnull()].copy()\n",
    "target = train_label\n",
    "test = data[data['isDefault'].isnull()].copy()\n",
    "\n",
    "target_encode_cols = ['postCode', 'regionCode', 'homeOwnership', 'employmentTitle','title']\n",
    "\n",
    "kflod_num = 5\n",
    "ss = 0.8\n",
    "fs = 0.4\n",
    "\n",
    "class_list = ['postCode', 'regionCode', 'homeOwnership', 'employmentTitle','title']\n",
    "MeanEnocodeFeature = class_list  # 声明需要平均数编码的特征\n",
    "ME = MeanEncoder(MeanEnocodeFeature, target_type='classification')  # 声明平均数编码的类\n",
    "train = ME.fit_transform(train, target)  # 对训练数据集的X和y进行拟合\n",
    "# x_train_fav = ME.fit_transform(x_train,y_train_fav)#对训练数据集的X和y进行拟合\n",
    "test = ME.transform(test)  # 对测试集进行编码\n",
    "print('num0:mean_encode train.shape', train.shape, test.shape)\n",
    "\n",
    "train, test = kfold_stats_feature(train, test, target_encode_cols, kflod_num)\n",
    "print('num1:target_encode train.shape', train.shape, test.shape)\n",
    "### target encoding目标编码，回归场景相对来说做目标编码的选择更多，不仅可以做均值编码，还可以做标准差编码、中位数编码等\n",
    "enc_cols = []\n",
    "stats_default_dict = {\n",
    "    'max': train['isDefault'].max(),\n",
    "    'min': train['isDefault'].min(),\n",
    "    'median': train['isDefault'].median(),\n",
    "    'mean': train['isDefault'].mean(),\n",
    "    'sum': train['isDefault'].sum(),\n",
    "    'std': train['isDefault'].std(),\n",
    "    'skew': train['isDefault'].skew(),\n",
    "    'kurt': train['isDefault'].kurt(),\n",
    "    'mad': train['isDefault'].mad()\n",
    "}\n",
    "### 暂且选择这三种编码\n",
    "enc_stats = ['max', 'min', 'skew', 'std']\n",
    "skf = KFold(n_splits=kflod_num, shuffle=True, random_state=6666)\n",
    "for f in tqdm(['postCode', 'regionCode', 'homeOwnership', 'employmentTitle','title']):\n",
    "    enc_dict = {}\n",
    "    for stat in enc_stats:\n",
    "        enc_dict['{}_target_{}'.format(f, stat)] = stat\n",
    "        train['{}_target_{}'.format(f, stat)] = 0\n",
    "        test['{}_target_{}'.format(f, stat)] = 0\n",
    "        enc_cols.append('{}_target_{}'.format(f, stat))\n",
    "    for i, (trn_idx, val_idx) in enumerate(skf.split(train, target)):\n",
    "        trn_x, val_x = train.iloc[trn_idx].reset_index(drop=True), train.iloc[val_idx].reset_index(drop=True)\n",
    "        enc_df = trn_x.groupby(f, as_index=False)['isDefault'].agg(enc_dict)\n",
    "        val_x = val_x[[f]].merge(enc_df, on=f, how='left')\n",
    "        test_x = test[[f]].merge(enc_df, on=f, how='left')\n",
    "        for stat in enc_stats:\n",
    "            val_x['{}_target_{}'.format(f, stat)] = val_x['{}_target_{}'.format(f, stat)].fillna(\n",
    "                stats_default_dict[stat])\n",
    "            test_x['{}_target_{}'.format(f, stat)] = test_x['{}_target_{}'.format(f, stat)].fillna(\n",
    "                stats_default_dict[stat])\n",
    "            train.loc[val_idx, '{}_target_{}'.format(f, stat)] = val_x['{}_target_{}'.format(f, stat)].values\n",
    "            test['{}_target_{}'.format(f, stat)] += test_x['{}_target_{}'.format(f, stat)].values / skf.n_splits\n",
    "\n",
    "print('num2:target_encode train.shape', train.shape, test.shape)\n",
    "\n",
    "train.drop(['postCode', 'regionCode', 'homeOwnership', 'employmentTitle','title'], axis=1, inplace=True)\n",
    "test.drop(['postCode', 'regionCode', 'homeOwnership', 'employmentTitle','title'], axis=1, inplace=True)\n",
    "print('输入数据维度：', train.shape, test.shape)\n",
    "\n",
    "xgb_preds, xgb_oof, xgb_score, feaNum = xgb_model(train=train, target=target, test=test, k=kflod_num)\n",
    "\n",
    "lgb_score = round(xgb_score, 5)\n",
    "sub_df = test[['id']].copy()\n",
    "sub_df['isDefault'] = xgb_preds\n",
    "off = test[['id']].copy()\n",
    "subVal_df = train[['id']].copy()\n",
    "subVal_df['isDefault'] = xgb_oof\n",
    "outpath = '../user_data/'\n",
    "\n",
    "all_auc_score = roc_auc_score(train_label, subVal_df['isDefault'])\n",
    "print('整体指标得分：', all_auc_score)\n",
    "all_auc_score = round(all_auc_score, 5)\n",
    "\n",
    "sub_df.to_csv(outpath+'xgb1.csv',index=False)\n",
    "subVal_df.to_csv(outpath+'xgb1Val.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "uuid": "5e11ae06-11ac-4726-9d6b-7ecf48825a84"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}